package zhiHu;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;

/**
 * Created by sheny on 2017/1/11.
 * 使用httpclient来建立链接
 */
public class SpiderByHttpClient {
    private static HttpClient httpClient = new HttpClient();

    /**
     * 根据url生成需要生成的文件名
     */
    private String getFilenameByPath(String url,String contentType){
        //移除http://7个字符
        url = url.substring(7);
        //确认抓去到的页面为html类型
        if(contentType.indexOf("html") != -1){
            //把url中所有特殊符号转为下划线_
            url = url.replaceAll("[\\?/:*|<>\"]","_")+".html";
        }else{
            url = url.replaceAll("[\\?/:*|<>\"]", "_") + "."
                    + contentType.substring(contentType.lastIndexOf("/") + 1);
        }
        return url;
    }
    /**
     * 保存网页字节数组到本地文件，filePath 为要保存的文件的相对地址
     */
    private void saveToLocal(byte[] data, String filePath) {
        try {
            DataOutputStream out = new DataOutputStream(new FileOutputStream(
                    new File(filePath)));
            for (int i = 0; i < data.length; i++){
                out.write(data[i]);
            }
            out.flush();
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 下载url指向的页面
     */
    public  String downLoadFile(String url){
        String filePath = null;
        // 1.设置httpClient参数 设置连接超时5s
        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
        // 2.生成getMethod 设置参数
        GetMethod getMethod = new GetMethod(url);
        // 设置get 请求超时5s
        getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
        // 设置请求重试处理
        getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,new DefaultHttpMethodRetryHandler());
        // 3.执行get请求
        try{
            int statusCode = httpClient.executeMethod(getMethod);
            // 判断访问状态吗
            if(statusCode != HttpStatus.SC_OK){
                System.err.println("Method failed: "
                        + getMethod.getStatusLine());
                filePath = null;
            }
            // 4.处理http响应内容
            byte[] responseBody = getMethod.getResponseBody();
            // 根据url 生成保存的文件名
            filePath = getFilenameByPath(url,getMethod.getResponseHeader("Content-Type").getValue());
            saveToLocal(responseBody,filePath);
        }catch (HttpException e) {
            // 发生致命的异常，可能是协议不对或者返回的内容有问题
            System.out.println("请检查你的http地址是否正确");
            e.printStackTrace();
        } catch (IOException e) {
            // 发生网络异常
            e.printStackTrace();
        } finally {
            // 释放连接
            getMethod.releaseConnection();
        }
        return filePath;
    }


}
